﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Xml;
using Common.Utilities.Dispatcher;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// 
    /// </summary>
    public class UnsupervisedClusteringWorker:IWorker
    {
        #region fields
        private Thread _WorkerThread;
        private UnsupervisedClusteringInstruction _Instruction;
        private UnsupervisedClusteringOutput _Output;
        private Guid _JobTicketID;
        private RunContext _Context;
        private string _WorkflowName;
        private string _ComponentName;

        /// <summary>
        /// 
        /// </summary>
        public string ComponentName
        {
            get { return _ComponentName; }
        }

        /// <summary>
        /// 
        /// </summary>
        public string WorkflowName
        {
            get { return _WorkflowName; }
        }
        #endregion

        #region IWorker Members

        public Thread WorkerThread
        {
            get { return _WorkerThread; }
        }

        public InstructionBase Instruction
        {
            get { return _Instruction; }
        }

        public ResultBase Output
        {
            get { return _Output; }
        }

        public Type ComponentType
        {
            get { return typeof(UnsupervisedClusteringComponent); }
        }

        public RunContext Context
        {
            get { return _Context; }
            set { _Context = value; }
        }

        public void SetInstruction(Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._JobTicketID = ticketGuid;
            this._Instruction = (UnsupervisedClusteringInstruction)request;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }

        public void SetInstruction(RunContext context, Guid ticketGuid, InstructionBase request, string workflowName, string componentName)
        {
            this._JobTicketID = ticketGuid;
            this._Instruction = (UnsupervisedClusteringInstruction)request;
            this._Context = context;
            this._WorkflowName = workflowName;
            this._ComponentName = componentName;
        }

        public void DoWork()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._JobTicketID, this, this.GetType(), this._Instruction);
            }
            this._WorkerThread = Thread.CurrentThread;
            this.ClusterDocuments();
        }

        public void DoWorkAsync()
        {
            if (this.WorkerStartedEvent != null)
            {
                this.WorkerStartedEvent(this._JobTicketID, this, this.GetType(), this._Instruction);
            }
            this._WorkerThread = new Thread(new ThreadStart(this.ClusterDocuments));
            this._WorkerThread.Start();
        }

        public void CancelWork()
        {
            if (this._WorkerThread != null && this._WorkerThread.IsAlive)
            {
                this._WorkerThread.Abort();
                if (this.WorkerCancelledEvent != null)
                {
                    this.WorkerCancelledEvent(this._JobTicketID, this.GetType(), this._Instruction);
                }
            }

        }

        public event WorkerStarted WorkerStartedEvent;
        public event WorkerFinished WorkerFinishedEvent;
        public event WorkerHasError WorkerErredEvent;
        public event WorkerCancelled WorkerCancelledEvent;
        public event WorkerStatusChanged WorkerStatusChangedEvent;

        #endregion

        #region work
        private void ClusterDocuments()
        {
            try
            {
                IDistanceCalculator distanceCalc = null;
                switch(this._Instruction.DistanceMethod)
                {
                    case DistanceCalculationMethod.CosineDistance:
                        distanceCalc = new CosineDistanceCalculator();
                        break;
                    case DistanceCalculationMethod.Euclidean:
                        distanceCalc = new EuclideanDistanceCalculator();
                        break;
                    case DistanceCalculationMethod.PearsonCoefficient:
                        distanceCalc = new PearsonCorrelationCalculator();
                        break;
                    case DistanceCalculationMethod.TanimotoCoefficient:
                        distanceCalc = new TanimotoCoefficientCalculator();
                        break;
                    case DistanceCalculationMethod.WeightedSum:
                        distanceCalc = new WeightedTermSimilarityCalculator();
                        break;
                    default:
                        distanceCalc = new CosineDistanceCalculator();
                        break;
                }

                IDocumentCluster clusterWorker = null;
                switch(this._Instruction.ClusterMethod)
                {
                    case DocumentClusteringMethod.KMeans:
                        clusterWorker = new KMeansCluster();
                        break;
                    case DocumentClusteringMethod.HierarchicalClustering:
                        clusterWorker = new HierarchicalCluster();
                        break;
                    case DocumentClusteringMethod.FrequentTermSet:
                        clusterWorker = new FrequentTermSetCluster();
                        break;
                    case DocumentClusteringMethod.KNearestNeighbor:
                        clusterWorker = new KNearestNeighborCluster();
                        break;
                    default:
                        clusterWorker = new KMeansCluster();
                        break;
                }
                clusterWorker.DistanceCalculator = distanceCalc;
                clusterWorker.PurityCalculator = new EntropyPurityCalculator();
                clusterWorker.CanOverlap = this._Instruction.CanOverlap;
                clusterWorker.ReassociationRange = this._Instruction.ReassociationDistanceRange;
                Dictionary<int, Dictionary<int, DocTermFreq>> docTerms = this.ReadDocumentTerms(this._Instruction.DocTermListFilePath);
                Dictionary<int, double> termWeights = this.ReadTermWeights(this._Instruction.DocTermListFilePath);
                List<Term> terms =new List<Term>();
                foreach(int docID in docTerms.Keys)
                {
                    Dictionary<int, DocTermFreq> termFreqs = docTerms[docID];
                    foreach(int termID in termFreqs.Keys)
                    {
                        DocTermFreq docTermFreq = termFreqs[termID];
                        Term term=new Term();
                        term.TermID = termID;
                        term.Phrase = docTermFreq.Phrase;
                        term.Weight = docTermFreq.Weight;
                        if (!terms.Contains(term))
                            terms.Add(term);
                    }
                }
                

                clusterWorker.OnClusterStatusChanged += new ClusterStatusChanged(clusterWorker_OnClusterStatusChanged);

                
                if(this._Instruction.TermFreqThreshold>0 && this._Instruction.ExcludedTerms !=null && 
                    this._Instruction.ExcludedTerms.Count>0)
                {
                    TfIdfTermFilter termFilter=new TfIdfTermFilter();
                    List<int> excludedTermIDs=new List<int>();
                    Dictionary<string,int> termPhraseIDMappings=new Dictionary<string, int>();
                    foreach(int docID in docTerms.Keys)
                    {
                        Dictionary<int, DocTermFreq> docTermFreqs = docTerms[docID];
                        foreach(int termID in docTermFreqs.Keys)
                        {
                            if(!termPhraseIDMappings.ContainsKey(docTermFreqs[termID].Phrase.ToLower()))
                            {
                                termPhraseIDMappings.Add(docTermFreqs[termID].Phrase.ToLower(),termID);
                            }
                        }
                    }
                    foreach(string exTerm in this._Instruction.ExcludedTerms)
                    {
                        if(termPhraseIDMappings.ContainsKey(exTerm.ToLower()))
                        {
                            excludedTermIDs.Add(termPhraseIDMappings[exTerm.ToLower()]);
                        }
                    }
                    termFilter.ExcludedTermIDs = excludedTermIDs;
                    termFilter.TermFrequencyThreshold = this._Instruction.TermFreqThreshold;
                    docTerms = termFilter.FilterDocumentTerms(docTerms);
                }


                Dictionary<int, ClusterNode> clusterNodes;
                if(this._Instruction.TermFuzzyMatch==TermFuzzyMatchMethod.None)
                {
                    clusterNodes = clusterWorker.ClusterDocuments(docTerms,false,0);
                }
                else if(this._Instruction.TermFuzzyMatch==TermFuzzyMatchMethod.TriGram)
                {
                    clusterNodes = clusterWorker.ClusterDocuments(docTerms,true,3);
                }
                else if (this._Instruction.TermFuzzyMatch == TermFuzzyMatchMethod.QuadGram)
                {
                    clusterNodes = clusterWorker.ClusterDocuments(docTerms, true, 4);
                }
                else
                {
                    Dictionary<string, List<int>> termStemmings2 = new Dictionary<string, List<int>>();
                    if (this._Instruction.TermFuzzyMatch == TermFuzzyMatchMethod.PorterStemming)
                        termStemmings2 = TermSimilarityMatrix.BuildTermPorterStemmingMappings(terms);
                    else
                        termStemmings2 = TermSimilarityMatrix.BuildTermInflectionalStemmingMappings(terms);

                    Dictionary<int, string> termStemmings1 = new Dictionary<int, string>();
                    foreach (string wordRoot in termStemmings2.Keys)
                    {
                        List<int> similarTermIDs = termStemmings2[wordRoot];
                        foreach (int similarTermID in similarTermIDs)
                        {
                            if (!termStemmings1.ContainsKey(similarTermID))
                                termStemmings1.Add(similarTermID, wordRoot);
                        }
                    }
                    clusterWorker.TermWeights = termWeights;
                    clusterWorker.CanOverlap = this._Instruction.CanOverlap;
                    clusterWorker.ReassociationRange = this._Instruction.ReassociationDistanceRange;
                    clusterWorker.SetThreshold(this._Instruction.TermCoverageThreshold,
                                               this._Instruction.TermFreqThreshold,
                                               this._Instruction.DistanceThreshold);
                    clusterNodes = clusterWorker.ClusterDocuments(docTerms, termStemmings1, termStemmings2);
                }
                
                Dictionary<int, string> docLabels = this.ReadDocumentLabels(this._Instruction.DocTermListFilePath);
                Dictionary<int, Term> allTerms = new Dictionary<int, Term>();
                foreach (int docID in docTerms.Keys)
                {
                    Dictionary<int, DocTermFreq> docTermFreqs = docTerms[docID];
                    foreach (int termID in docTermFreqs.Keys)
                    {
                        if(!allTerms.ContainsKey(termID))
                        {
                            Term term=new Term();
                            term.TermID = termID;
                            term.Phrase = docTermFreqs[termID].Phrase;
                            term.Weight = docTermFreqs[termID].Weight;
                            allTerms.Add(termID, term);
                        }
                    }
                }
                this.SaveClusterResult(docTerms, clusterNodes, docLabels,
                                       allTerms, this._Instruction.DocTermListFilePath,
                                       this._Instruction.ClusteringResultFilePath);

                this._Output = 
                    new UnsupervisedClusteringOutput(
                        this._Instruction.OwnerComponentID,
                        this._JobTicketID, 
                        this._Instruction.ClusteringResultFilePath,
                        clusterNodes.Count);

                if(this.WorkerFinishedEvent !=null)
                {
                    this.WorkerFinishedEvent(this._JobTicketID, this.GetType(), this._Instruction, this._Output);
                }
            }
            catch (Exception ex)
            {
                if(this.WorkerErredEvent !=null)
                {
                    this.WorkerErredEvent(this._JobTicketID, this.GetType(), this._Instruction, ex.ToString(),true);
                }
            }
        }

        private Dictionary<int, string> ReadDocumentLabels(string path)
        {
            Dictionary<int,string> docLabels=new Dictionary<int, string>();
            XmlDocument xDoc = new XmlDocument();
            xDoc.Load(path);
            XmlNodeList docNodes = xDoc.SelectNodes("//Doc");
            foreach(XmlNode docNode in docNodes)
            {
                int docID = int.Parse(docNode.Attributes.GetNamedItem("DocID").Value);
                string docLabel = docNode.Attributes.GetNamedItem("Label").Value;
                docLabels.Add(docID,docLabel);
            }
            return docLabels;
        }

        void clusterWorker_OnClusterStatusChanged(int clusterCount, int remainingDocCount, string message, int pct)
        {
            if(this.WorkerStatusChangedEvent!=null)
            {
                string msg = string.Format("{0} Cluster={1}, remaining docs={2}",
                    message, clusterCount, remainingDocCount);
                this.WorkerStatusChangedEvent(this._JobTicketID, this.GetType(), this._Instruction, this.WorkflowName, this.ComponentName, msg, pct);
            }
        }

        private Dictionary<int,Dictionary<int,DocTermFreq>> ReadDocumentTerms(string docTermListFilePath)
        {
            Dictionary<int, Dictionary<int, DocTermFreq>> docTerms = new Dictionary<int, Dictionary<int, DocTermFreq>>();
            XmlDocument xDoc =new XmlDocument();
            xDoc.Load(docTermListFilePath);
            XmlNodeList docNodes = xDoc.SelectNodes("//Doc");
            foreach(XmlNode docNode in docNodes)
            {
                int docID = int.Parse(docNode.Attributes.GetNamedItem("DocID").Value);
                XmlNodeList termNodes = docNode.SelectNodes("Term");
                Dictionary<int, DocTermFreq> terms = new Dictionary<int, DocTermFreq>();
                foreach(XmlNode termNode in termNodes)
                {
                    DocTermFreq docTermFreq=new DocTermFreq();
                    docTermFreq.TermID = int.Parse(termNode.Attributes.GetNamedItem("TermID").Value);
                    docTermFreq.Phrase = termNode.Attributes.GetNamedItem("Phrase").Value;
                    docTermFreq.Count = int.Parse(termNode.Attributes.GetNamedItem("Count").Value);
                    docTermFreq.Weight = double.Parse(termNode.Attributes.GetNamedItem("Weight").Value);
                    terms.Add(docTermFreq.TermID,docTermFreq);
                }
                docTerms.Add(docID,terms);
            }
            return docTerms;
        }

        private Dictionary<int,double> ReadTermWeights(string docTermListFilePath)
        {
            Dictionary<int,double> termWeights=new Dictionary<int, double>();
            XmlDocument xDoc = new XmlDocument();
            xDoc.Load(docTermListFilePath);
            XmlNodeList termNodes = xDoc.SelectNodes("//Term");
            foreach(XmlNode termNode in termNodes)
            {
                int termID = int.Parse(termNode.Attributes.GetNamedItem("TermID").Value);
                double termWeight = double.Parse(termNode.Attributes.GetNamedItem("Weight").Value);
                // if (termWeight != 0)
                {
                    if (!termWeights.ContainsKey(termID))
                    {
                        termWeights.Add(termID, termWeight);
                    }
                    else if (termWeights[termID] < termWeight)
                    {
                        termWeights[termID] = termWeight;
                    }
                }
            }
            return termWeights;
        }

        private void SaveClusterResult(
            Dictionary<int,Dictionary<int,DocTermFreq>> docTerms, 
            Dictionary<int,ClusterNode> clusterNodes, 
            Dictionary<int,string> docLabels,
            Dictionary<int,Term> allTerms,
            string docTermListFilePath,
            string resultFilePath)
        {
            if(File.Exists(resultFilePath))
            {
                File.Delete(resultFilePath);
            }

            XmlTextWriter xWriter =new XmlTextWriter(resultFilePath, Encoding.UTF8);
            xWriter.Indentation = 4;
			// Clusters
			xWriter.WriteStartElement("Clusters");
            xWriter.WriteAttributeString("DocTermListFilePath", docTermListFilePath);
            xWriter.WriteAttributeString("ClusterCount", clusterNodes.Count.ToString());
            xWriter.WriteAttributeString("DocCount",docTerms.Count.ToString());
            xWriter.WriteAttributeString("TermCount",allTerms.Count.ToString());
            
            foreach(int clusterID in clusterNodes.Keys)
            {
				// Cluster 
                xWriter.WriteStartElement("Cluster");
                xWriter.WriteAttributeString("ClusterID",clusterID.ToString());
                ClusterNode clusterNode = clusterNodes[clusterID];
                xWriter.WriteAttributeString("Distance",clusterNode.Distance.ToString());
                
                // shared terms
                xWriter.WriteStartElement("SharedTerms");
                foreach(int sharedTermID in clusterNode.SupportiveTermWeights.Keys)
                {
                    xWriter.WriteStartElement("Term");
                    xWriter.WriteAttributeString("TermID", sharedTermID.ToString());
					xWriter.WriteAttributeString("Term", allTerms[sharedTermID].Phrase);
                    xWriter.WriteAttributeString("Weight",clusterNode.SupportiveTermWeights[sharedTermID].ToString());
                    xWriter.WriteEndElement();  // Term
                }
                xWriter.WriteEndElement();  // SharedTerms

                if (clusterNode.DocIDs != null && clusterNode.DocIDs.Count > 0)
                {
                    xWriter.WriteStartElement("Docs");
                    xWriter.WriteAttributeString("DocCount", clusterNode.DocIDs.Count.ToString());
                    foreach (int docID in clusterNode.DocIDs)
                    {
                        xWriter.WriteStartElement("Doc");
                        xWriter.WriteAttributeString("DocID", docID.ToString());
                    	xWriter.WriteAttributeString("Label",
                    	                             docLabels.ContainsKey(docID) ? docLabels[docID] : string.Empty);
                    	Dictionary<int, DocTermFreq> termsForDoc = docTerms[docID];
						foreach (int termID in termsForDoc.Keys)
						{
							xWriter.WriteStartElement("Term");	// term
							xWriter.WriteAttributeString("TermID",termID.ToString());
							xWriter.WriteAttributeString("Phrase",termsForDoc[termID].Phrase);
							xWriter.WriteAttributeString("Weight",termsForDoc[termID].Weight.ToString());
							xWriter.WriteAttributeString("Count",termsForDoc[termID].Count.ToString());
							xWriter.WriteEndElement();
						}
                        xWriter.WriteEndElement(); // Doc
                    }
                    xWriter.WriteEndElement(); // Docs
                }
                if (clusterNode.ChildClusterIDs != null && clusterNode.ChildClusterIDs.Count > 0)
                {
                    xWriter.WriteStartElement("ChildClusters");
                    foreach (int childClusterID in clusterNode.ChildClusterIDs)
                    {
                        xWriter.WriteStartElement("Cluster");
                        xWriter.WriteAttributeString("ClusterID", childClusterID.ToString());
                        xWriter.WriteEndElement(); // Cluster
                    }
                    xWriter.WriteEndElement(); // ChildClusters
                }
                xWriter.WriteEndElement();  // Cluster
            }
            xWriter.WriteEndElement(); // Clusters

            xWriter.Flush();
            xWriter.Close();
        }
        #endregion
    }
}
